package com.googlecode.clearnlp.tokenization;

import java.io.BufferedReader;
import java.util.List;

public interface Tokenizer {

	/**
	 * Returns a list of token in the specific reader.
	 * @param fin the reader to retrieve tokens from.
	 * @return a list of token in the specific reader.
	 */
	public abstract List<Token> getTokens(BufferedReader fin);

	/**
	 * Returns a list of tokens from the specific string.
	 * @param str the string to retrieve tokens from.
	 * @return a list of tokens from the specific string.
	 */
	public abstract List<Token> getTokens(String str);

}